{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv('engineered_data_train.csv')\n",
    "data_test = pd.read_csv('engineered_data_test.csv')\n",
    "\n",
    "X_train = data_train.drop('Y', axis=1)\n",
    "y_train = data_train['Y']\n",
    "\n",
    "X_test = data_test.drop('Y', axis=1)\n",
    "y_test = data_test['Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replace with CV："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train_valid, X_test_valid, y_train_valid, y_test_valid = train_test_split(X_train, y_train, test_size=0.3, random_state=49)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "lr=LogisticRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:462: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\scipy\\optimize\\linesearch.py:313: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\optimize.py:203: ConvergenceWarning: newton-cg failed to converge. Increase the number of iterations.\n",
      "  \"number of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\sag.py:334: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "C:\\Users\\baoji\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\sag.py:334: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "solver=['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']\n",
    "maxAuc=0\n",
    "sol=\"\"\n",
    "for st in solver:\n",
    "    lr=LogisticRegression(solver=st).fit(X_train_valid, y_train_valid)\n",
    "    lr_probs = lr.predict_proba(X_test_valid)[:, 1]\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test_valid, lr_probs)\n",
    "    auc = metrics.auc(fpr, tpr)\n",
    "    if maxAuc<auc:\n",
    "        maxAuc=auc\n",
    "        sol=st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9926826941423395\n",
      "newton-cg\n"
     ]
    }
   ],
   "source": [
    "print(maxAuc)\n",
    "print(sol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "C_value=[0.01, 0.1, 1, 10, 100]\n",
    "maxAuc=0\n",
    "sol=0\n",
    "for st in C_value:\n",
    "    lr=LogisticRegression(solver='liblinear',max_iter=1000, C=st).fit(X_train_valid, y_train_valid)\n",
    "    lr_probs = lr.predict_proba(X_test_valid)[:, 1]\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test_valid, lr_probs)\n",
    "    auc = metrics.auc(fpr, tpr)\n",
    "    if maxAuc<auc:\n",
    "        maxAuc=auc\n",
    "        sol=st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9524686688561441\n",
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(maxAuc)\n",
    "print(sol)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=LogisticRegression(solver='lbfgs',max_iter=1000, C=0.1).fit(X_train_valid, y_train_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7091909155429382"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[8633, 1078],\n",
       "       [5478, 7355]], dtype=int64)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "p=lr.predict(X_test)\n",
    "confusion_matrix(y_test,p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_y=lr.predict(X_test_valid) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs_y=lr.predict_proba(X_test_valid) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XeYVPXVwPHvYalSpYl0VKT3pSugiMGuROydaNSIMZpE1Lx2YzT2iF2DBSkaC7FEbFhQhFURkC51AaVJk7pw3j/Ond1h3TILO3NnZs/neebZmbl35p47A/fMr4uq4pxzzgGUCzsA55xzycOTgnPOuVyeFJxzzuXypOCccy6XJwXnnHO5PCk455zL5UnB7UVEvheRAcXs01REtohIRoLCijsRWSIixwT3bxWRl0KMJSHHF5HmIqIiUn4fXjtARLKL2D5KRO7cj9hURA7b19e7fedJIUUEF61twcX4JxH5t4hUK+3jqGo7VZ1UzD7LVLWaqu4u7eMHF8RdwXluEJEvRKR3aR8nTMG5RW57or7XLSJybtjxJYKIHCwiz4rIKhHZLCJzReQ2Eakap+PlJn1XNE8KqeUkVa0GdAW6A3/Lv4OYVP9exwXnWRf4GHgl5HhKVZBQqwXnuIzgew1uo0vyXvvyKz9sIlIb+BKoAvRW1erAIKAWcGgpHyvlPp+wpfrFo0xS1RXAu0B7ABGZJCJ3ichkYCtwiIjUjPoltkJE7oyu7hGRS0VkTvArbbaIdA2ej65G6SEiWSKyKSidPBA8v1e1g4g0FJEJIrJeRBaKyKVRx7lVRMaLyAvBsb4XkcwYzzMHGA00EpF6Ue95oohMjypJdIza1kREXhORNSKyTkQeDZ4/VEQ+Cp5bKyKjRaRWST/74DM7Mepx+eD9uopIZRF5KTjGBhGZJiIHlfQYgYqFfWbBd3S9iMwAfgliaCgi/wnOe7GIXB21f4HfY5RzRWRZcB43Rb2ukog8JCIrg9tDIlKpkM+li4h8E8Q7DqhcxLldC2wGzlPVJQCqulxV/6iqM6L2O0ZEFojIzyIyUkQkOFaR32UBn88YoCnwX7HS2F+L/OTLOE8KKUhEmgDHA99GPX0+cBlQHVgKPA/kAIcBXYBjgd8Frx8K3ApcANQATgbWFXCoh4GHVbUG9gtufCEhjQGygYbA6cDfRWRg1PaTgbHYL8EJwKMxnmfFIMZ1wM/Bc12B54DfA3WAJ4EJwQUsA3grOP/mQKPguAAC3B3E2AZoEnwGJTUGODvq8W+Atar6DXAhUDN47zrA5cC2fTgGFP+ZnQ2cEGzfA/wX+A4754HANSLym2Df4r7HI4BWwetuFpE2wfM3Ab2AzkAnoAcFl04rAm8ALwK1sZLdb4s4t2OA11R1TxH7AJyIlYg7AWdgnzXE9l3mfj6qejZ7l8juLea4ZZuq+i0FbsASYAuwAbvoPQZUCbZNAm6P2vcgYEdke/Dc2cDHwf33gD8WcZxjgvufArcBdfPt0xxQoDz2H3I3UD1q+93AqOD+rcAHUdvaAtuKOM9bgZ3Bee7GEsKAqO2PA3fke808oD/QG1gDlI/h8zwV+LaQ874VeKmQ1x2G/co9IHg8Grg5uH8J8AXQsYTf6zEFfAaFfmbBay6JetwTWJbvPW4A/h3j99g46rmpwFnB/R+A46O2/QZYEtwfAGQH9/sBKwGJ2vcL4M5CznkBcHkxn4sCR0Q9Hg+MKMF3eUm+fX71Ofut4JuXFFLLqapaS1WbqeqVqhr9K3R51P1mQAVgVVCNsQH7RV0/2N4E+w9fnGHA4cDcoCrkxAL2aQisV9XNUc8txX6xRvwYdX8rUDmo8jhX8hpY343aZ7yq1sKS2yygW75zuy5yXsG5NQniaAIsVat22ouI1BeRsWJVaZuAl7A2ixJR1YXAHOAkETkA+0X/crD5RSzhjg2qW+4VkQolPUagwM8s6rn833fDfJ/JjdjnB8V/j/mPFenA0BD7LiOWBs/l1xBYocHVN2rfwqwDDi5ie5FxxfhdLsftE08K6SP6P+RyrKRQN0gitVS1hqq2i9pebIOeqi5QK3rXB+4BXpVf9w5ZCdQWkepRzzUFVsTw/qM1r4H1uAK2r8WqiW4VkchFZDlwV9R51VLVA1R1TLCtqRTcuHg39hl1VKtGOQ+rhtgXkSqkU4DZQaJAVXep6m2q2hbog1V/XLCPxyhO/u97cb7PpLqqHh/EFcv3WJCVWMKJaBo8l98qrN1H8u1bmA+A02TfO0TE8l3mn/7Zp4OOkSeFNKSqq4CJwP0iUkNEygWNc/2DXZ4B/iwi3cQcJiLN8r+PiJwnIvXU6n43BE/v1Q1VVZdjVQV3Bw2tHbFfpiXqRVPEuczFfn1HGgefBi4XkZ5B7FVF5IQgKU3FLlD/CJ6vLCJ9g9dVJ6h+E5FGwF/2I6yxWBvNFeSVEhCRo0SkQ9C2sQnYRb7PK06mApuCxtUqIpIhIu1FpHsQV7HfYyHGAH8TkXoiUhe4GftVnt+XWPvV1UEJcAjW/lCYB7C2rOcj/+5EpJGIPCBRnQaKsC/f5U/AITHsV+Z5UkhfFwAVgdlYI+2rBEV2VX0FuAu7oG3GGglrF/Aeg4HvRWQL1lh5lqpuL2C/s7H66ZXA68Atqvp+KZ7LP4HLRKS+qmYBl2INrz8DC4GLANTGTZyE1fsvwxq/zwze4zasK+9G4G3gtX0NJki6X2KlgXFRmxpgn/MmrIrpEwq+iJaqqPPuDCwG1mKJv2awS6zfY353AlnADGAm8E3wXP7j7wSGYN/Dz9hnXujnq6rrsc9uF/CViGwGPsS+m4UxxLUv3+XdWILbICJ/jmH/Mkv2rgZ0zjlXlnlJwTnnXK64JQUReU5EVovIrEK2i4g8IjbYaUbQ/9w551yI4llSGIXVZRbmOKBlcLsM63/unHMuRHFLCqr6KbC+iF1OAV5QMwWoFdXt0DnnXAjCnCyqEXsPMMkOnluVf0cRuQwrTVC1atVurVu3LvnRtm2DZcugenWoVQuqVAHZ127qzrlUpQq7dkHFivZ43TrYsQP27LFtqlCpEhwUDP1bssS2R14LUK0aNG5s9+fMsfdThZwcKFcOateGZkEn72+/zXvfiPr1oUkTO+a30ZPVBBo0gEaN7P1WrYLVq+2S1bbtvp/3119/vVZV6xW3X5hJoaArcoFdoVT1KeApgMzMTM3Kyir50ZYvhyOOsMSwahVUrgy9e8Ozz0KLFiV/P+dcaHbuzLuoT50Ks2bB4sWwcSNs3mwX0Mces+1//St88olduDdtsktA27YQuYz07g1ffWWXhAoV7H0zM+H112370KHw009QvjxkZNhFv3dvuPVW2/6HP8DWrbBoEXTsaPt17w7nnGPb//Y32L3bXlepEtStC926Qc+eFtOoUbY9Jyfv1rMnHHkkTJgAp5yS9z433rjvn5mIFDXKPFeYSSEbm5YgojEFj5YsHU2awNKllhy+/BK++AImT7ZvaetW+Pe/7V9Cz55xC8E5VzI7d8ITT8ADD9hFe8kS+y+7Y4cV/kXg8cftwlquHNSoYZUBB0dVRFeuDAceaBf7qlVhyBDo0cN+uYvA++/DAQfY6wvySjETt48cWfT2O4tYaqhCBbj00sK3n3yy/Yb95hsYMKDo45SWuI5TEJHmwFuq2r6AbScAV2GzffYEHlHVokZBAvtRUijKL79YWS8nB0491ZJDt27Qtav9C3TOJdTEiVaF0rIlDB8OM2ZAzZr22656dbvI33STXVSzsy15NGlij13BRORrVS122vq4lRSCOcwHAHXFlu27BZukDVV9AngHSwgLscmuLo5XLMWqWhXGjIF77oF33oHnn7fnlyyxZDF5MqxcCUcfDXXqhBamc2XBY49ZlQxYwf6ZZ4reP1K370pHyo1ojktJIZqqVQ7Ong0nnmjly3POsaRRvry1S7RqZX/POy9+cThXBs2dC23aWAF9/nxoWtS0eq5EYi0peFKIRU4OTJkCr71m7RFz58Lhh1vrFMBf/gJbtsDxx8NJJyU2NufShKo11M6aZSUELwGUrtCrj9JKpIRwxBH2eM8e68cG1m3gq6/gu++sRax3b/uXfd55tv+uXbB9u/Vh8y6wzhVqyRIrpN94oyeEMPncR/uiXDmoF3T3zciATz+FtWvhttts2/jxVvYFSxY1atitY0dryL7ySvs5BHndKJwr41q0sEbjm28OO5KyzauP4kHVShMZGbBiBYwebX8XLbLO1CtWWOIYNAjeeMP6yDVubF0tWrWykS9XXgkNG1rH6q1bbbRLYX3mnEtxH31kXS79n3j8eJtCsot0kp49G159FRYsgHnz4IcfYMMGmDnTRtg89BD86U/Wkfqww6xK6qijrBG8cuWwz8K5/fbEE3DFFdbf/8orw44mfXlSSGWR70TExsB/8QUsXGgJZPJk65S9YYMlittvt+dat4ZOnaBdO7tfs2bRx3AuZLt2wb332kjdPn3gww/9d048eUNzKotukO7SxW4Ru3ZZieKAA/L2Xb/eOnNv3WrPNW1qo7cBHn7YekbVr28J45BDbFIXb/R2IZoxA84915rWhg6FF1/0caLJwpNCqqlQAdpHDRD/v/+z2+7dVvU0d671dooYN8660UY7/nh4+227f+ONNv6/fn0bqJeZmTcTmHOlZOdOm06iZk2rAa1d2/pXvPGGTeXgv1GShyeFdJGRYWMnDj987+e/+ML+9/30k7VTLF6896js0aOtU3h0NeItt9hsX7t2WZVVhw7eAuhKbNMm66397rvw0kuwZo39nvnuO+tXsWCBJ4Nk5EmhLKhSBZo3t1t+S5fa4LzVqy1hvP++tU2A9Q/s3NmmAWnWzKqlGjaE3/3OxmNE5guOTFfpyqwff7TfHzNn2m8KsOqht96y+0OGwEUX7d3DyBNCcvKGZle4X36xUdxZWTbf8PLldnvySRtv8dFHcOyx1k7RurXd2rSBE06w6iiXdtatsyE3FSrYpHXPPmt9IRYssO3ly1sTVqVKNl319u3Qq5f3e0gG3tDs9l/VqnD++XaLFvkh0agR3HCDrTIydy68955VHk+bZknhP/+Bf/3LkkWrVnbr29evEEksshhMuXJWc/i//1kBcvZsu/34o329mZl2/5tv7HfAZZfZdNTdu+c1GPfvH+65uH3jScGVXKTc36oV3HFH3vM5OTZXQZMmefvl5NiE9OuDlVkzMqyqqnZtawDfutWqoiK9qVzCbN9uC9RMn263rCzL7S+9BGecYaWC666zGVratoXBg60DW4MG9voLLrCbSy+eFFzpKV/eBthFDBliN1W7wsycCZ9/bpPhAzz6KLz8stVFHHKINZJ37Zq3pNW2bdYe4vaLqjUdRS7+Xbtaj5/Vq/N+zdevb81HAwfmfYV9+sDPP1vBzuv/yw5vU3Dh+fln657yySc2OG/OHFtBJdKF9ogj7LlI1VPr1lZvMXBguHEnuQ0bbBnyPXvguOOsuufnn22bCPz5zzZoTNXaBTp1yvv179KXj2h2qSky/QfA00/D11/bYL25c60S+8QT4b//te1HHWV1G5Gk0aqV1XPUrRte/CH44Qe7uH/2mRXEmje3ORoBzjzTCmadOlkJoX17aypyZY83NLvUFF1Pcemley9gG1mVHexncP36VpJ4/32bbRZsEp3HHrMG7/PPt8bwpk2tlNG8uT2uXj1hpxMPmzZZDyCwBt6nn7b7Bx9si70ff3zevuPGJT4+l9o8KbjUUbNmXs+lcuXyrni7d1uX2Xnz8upBNm60vpJvvZU3/QfAXXfZKO5Nm6xNI9IzqmnTpE0WS5daCeDzz6030LJl1h5Qp45NtHv44XDaadYs43X/bn95UnCpLyPDJuNv0SLvuXr1bE0LVVvrYs4cG2PRs6dtz8qyld+j1aljPaWOOgpWrbK+mO3aJbQLrapVB9WubbcXX8zr4VOjhjX+XnGF5UGweYOcK02eFFx6E7EEEVkUKeLoo61FdsECSxirVllJo1Ej2/7223lVV7Vr21QfXbrYlJ7R04TsJ1UbCTxxonUPzcqyHPbss3DJJdY76JFHrM29Y0fLf87Fkzc0O1eQNWtsXe45c+yne1aW3V+50rr23H8/fPyxzdswbFheN9sYLFhgM4S0bWs9devWtYt9+/bWuapLFxswHslPzpUGb2h2bn/UqwcnnWS3iOgfUBkZVrJ4+20bVzFsmO17zDEFvl12tg3wfv1164Hbvz9MmmSFjjfesNziA71dMvCk4Fysoltxr7nGbl99ZQsd/etf1uPpmGNsFHezZra8avPmvL2gJQ990YNP6M8hrSpy1llw/fV5b3XKKYk/FecK40nBuf3Rs6eVFrZtg507ycmBj9/cilY4gX6bZlH5gw84YcXznAD8eM0/aPDg9VZnNGMG7O7njQQu6XhScK4UrP2lCg89VIVnnoGffqpBrVpP8e5YmyGUTZvgww9p0Lat7fzFFzbPxIEH2sCCHj2sW1Hv3r4epQudJwXn9tPmzTbcYf16a1a46CKbPC532qYaNWwgQcTRR1vX17fessbsCRPs+YUL4dBDrWFbxAbb+eJGLsE8KTi3j9avt96q1atbT9Wjj7Zuo8WqWhVOP91uYBMTTZmSN85ixAh49VV78z59bIjyiSfmzT7rXBx5l1TnSmjDBhg50gZHv/GGrTNUqhYsgA8/tJnsJk2CRYts8qLp0237009biaJ796Qdhe2Sj3dJda6Ubd0Kjz8Od99tbcUnnGAzZJS6li3tdvnl1g12zhyb1wLs8YgRVkwRsXmu+/a1UscJJ8QhGFfWeFJwLkaDBlkb8aBBlhi6dUvAQUVslFukkVrESg5ffmlDoL/5xno/NWtmSWHNGitVDBpkw6D79LEGD+/l5GLk1UfO5ZOTY1NPv/qqTT8xa5Z1CvrPf2yw2YABYUeYj6qNkahUyUoUl19us+etWWPba9SAsWNtcYUNG+zWrJnPnlfGePWRcyW0YIHNXvH663ZtPeAA60W0caMlhd/+NuwICyGStzBy/frw2muWKBYssBLFl19aGwRYI8jFF1t2y8y0PrPHHmvjLbw04fCSgivDdu6E996Dxo1tvqGsLOjXz4YQDB1qP6zTbunoxYut+JOVBZMn2+JFqjYhYIMG1n5Rp44lF5dWfOU15wqwZQu8+aaty/O//8FPP1lty+OP27Vx27Y0TARF2bDBusMOHmyPjzrKejxlZsJZZ8EZZ3hX2DThScE57EK/ejUcdJA97tgRZs7Maxu48EKrPYnUvpR5X31ls7+++qothQrwxz/CQw+FG5fbb0nRpiAig4GHgQzgGVX9R77tTYHngVrBPiNU9Z14xuTS28qVtlbx1Km28NqcOTZN9U8/WZX5TTdZdVHv3j5YuEA9e9ptxAgbYT12rC3tBrYE3JgxcM45tlKdS0txKymISAYwHxgEZAPTgLNVdXbUPk8B36rq4yLSFnhHVZsX9b5eUnARGzda1XhWFlx1lQ0Uvukm+PvfrWG4Y0dbOK17d1uwxksD++mVV6w6CWzxh9//3j7YMlXflrqSoaTQA1ioqouCgMYCpwCzo/ZRIFiCnJrAyjjG49LA0qXw/PPWQDxlCuzZY8/3728daS6+GIYMsYRQoUK4saadoUOt9DB+vPViGj7cMvDSpf5hp5F4FqAbAcujHmcHz0W7FThPRLKBd4DhBb2RiFwmIlkikrUm0vfapb2dO22w2N13W9snwI8/2po227fDjTdaclizJpiNFBvg262bX6Pi5tBD4YYbrO3h88/hD3/I+7Bfftkm83MpLZ4lhYJGxuSvqzobGKWq94tIb+BFEWmvqnv2epHqU8BTYNVHcYnWJYWcHBg1ygaKffqpTS0BcMst1jDcrZvN8FCrVphROsCm1+jb1+7Png3nnWct+926wXXXWVWTj31IOfFMCtlAdF+2xvy6emgYMBhAVb8UkcpAXWB1HONySSAytmraNLueVKoEN99s15C77rIfn8OGWSLo18/WMQYoX94TQlJq29aqkcaOhRdesMbo66+3aqauXcOOzpVAPJPCNKCliLQAVgBnAefk22cZMBAYJSJtgMqA1w+lmV27YPlyOOQQe3zVVfDSS9ZQDJYIIjONiliVUYMGPgtDymnSBP7yF7j2WksGzz9vE/uBDQrJybEvumLFcON0RYrrOAUROR54COtu+pyq3iUitwNZqjoh6HH0NFANq1r6q6pOLOo9vfdR8lu2zKp+vvjCurpPn24X+M2brQTwwAPWXtm1q/V+bNXKrxNpb+BA+Ogjm3PpwgttJaLI+hEuIXzwmkuYJUusIfi006BmTav++dvfbKr/bt2sS2jHjtZ5xbuFllE7dthKc088YWtFgPUYuPnmUMMqS5KhS6pLU5s2We3AW29Zm8CSJfZ83bq2QNgll9iylO3aeTujC1SqZDMK/va3Vpd4zz02tTfYVBt79thKcy50XlJwxYqMEm7a1EYCz5lj7YoNG9p0/f362ZQ57dp5O4DbB3/6k3Vnfe21vN5MrtR5ScHtM1Vb8XHyZGsT+P57e/53v7Ok0KqVjSLu0sWninCl4KKLbL7yI4+Eq6+2AXE+Sjo0nhTKuGXL8hbxUrVGYBF48EFbcrJHDzj7bPjNb2xBL7BEkJBVx1zZ0KmTzVI4YgQ8/LBN5/322173GBJPCmXIli1QrZrdv+MOePFFGysAVuV7/PF5+37+uVXxenWQS4jq1WHkSKuXvO8+GzHdp0/YUZVJXvhPYzk5Vgq45RYbI3DggdZIDDZhXJs2ViL4+mvrLvraa3mvrVPHE4ILwR/+YAsBeUIIjZcU0tTbb9usAxs2WHVPjx5w9NF5E8j95S92cy4pPf00XHpp2FGUSV5SSHGq1hB85502KdzYsfb84YfbbKHjxtmEcV9+Cc8841NEuBTwzTdw2WU2I6JLOC8ppKjNm23sz4QJNjpYxFZQjAwOa9kSnn021BCd2zfTp9vf7t3h3nutl4NLGC8ppABVmD8fHnnESgRgDcZffgnNm8OTT9p4oKlTbVSxcyntvPPgX/+ynhGDB8Nxx+U1hrm488FrSeyjj6yH0Icf2kUfrDQwdaqVDFS9MdilsR07LDl89pkNoY9MoFW9etiRpaRYB695SSGJzJlj43a2b7fHH35o1UM9esBjj9n6JdOm5SUCTwgurVWqBH/+M7z5pv1jX7nSFth+6KG8HhOu1HlJIUS7d9uSkhMn2oDOmTPt+U8+sakjNm2ygZ3lveXHOVt279JLbdKtI4+Exx+3uVVcTLykkKTWr4cVK+z+1Kk2J9idd9rsoo88Yj+G+vWz7TVqeEJwLleDBlZ0fu45+O476NDBFvNJsR+2yc6TQgLk5Ni4gaFD4eCDrdcQWBfSN96AtWut2nT4cNvunCuECFx8MSxaZPOzN2qUV4/6zTeeIEqB/w6NszvvhEcfhZ9+gnr14Mor7d802L/lU04JNz7nUlKdOnD77XmPp0yx2Rp797ZZV3/7W5+tcR/5p1bKVOHdd/N+sKxfbyP233zTqo0efNAWnHHOlaJOnWzupFWr4IwzoH17G+PgXVlLzBuaS0FOjjUWjxlj1UBLl1p10fHHe7dR5xJqzx4YPz5vzddVq6wX09SpNqinfv2wIwyNNzQnyIwZtu7wCSfYv8U2bWxR+kGDbLsnBOcSqFw5OOssSwJLllhCULUBcY0a2dKAkQVCXIE8KeyDHTusTQtsHM0ZZ1iX0o0brero3HMtUTjnQlSjhv0Vsf+g115rK0d17Gj/aT05FMiTQgls2WKNxq1bW0lg82Zo0cImnTv1VJuO2jmXhNq1s3Wh58+36YEnTrQVpsBWk9qyJdz4kkihSUFEZorIjAJuM0VkRiKDDNuqVXD99dCkiXUbbdDAZiP10fbOpZh69eAf/7ABQcccY8/dfTc0a2btEDt2hBtfEiiqS+qJCYsiyS1ZYotBnX669Xbr1SvsiJxz+yV6DeghQ2DWLLjuOrj/fuu1dM45ZbZBsNCkoKpLExlIslHdu+vz8uXQsGHYUTnnSl2fPtYYOHGiDYg77zxbpzYyyrSMKar6aLOIbCrgtllE0r7z72OPQd++1okBPCE4l9ZEbN2Gr76yueiHDbPnf/65zE2+5+MUCrBhg61p3KULfPBBmS1FOle27dlja9iq2sC49u3Djmi/lPo4BRGpLyJNI7f9Cy+53XST/UC47z5PCM6VWSJwwQW2ElzfvnnTGKe5YpOCiJwsIguAxcAnwBLg3TjHFZrt22H0aBg40EoKzrkySgQuucRGqFatalMURLqxprFYSgp3AL2A+araAhgITI5rVCHKyrLpqm+8MexInHNJoVkza4jevNlmtExzscySuktV14lIOREpp6ofi8g9cY8sJEccAatXe7WRcy5Kp07w3ns2f1Kai6WksEFEqgGfAqNF5GEgJ75hhWP3bmtTKlfOk4JzLp+ePeGgg2yA2wsvpO3aDbEkhVOArcCfgP8BPwAnxTOosIweDU2b2mA155wr0OjRcOGF8O9/hx1JXMSSFOoDFVU1R1WfB54G0nKCh6lTbf2Dpmndt8o5t18uvthGtN58M/zyS9jRlLpYksIrQPTojd3Bc2nn22+hWzdfsMk5VwQRmz9pxQobv5BmYrn8lVfVnZEHwf2Ksby5iAwWkXkislBERhSyzxkiMltEvheRl2MLu/Spwpw50LZtWBE451JGv37WRfUf/7DqhTQSS1JYIyInRx6IyCnA2uJeJCIZwEjgOKAtcLaItM23T0vgBqCvqrYDrilB7KVq5UobsJbigxadc4ly991WtbBmTdiRlKpYuqRejvU6GgkokA1cEMPregALVXURgIiMxRqtZ0ftcykwUlV/BlDV1SWIvVTt3m1rcAwYEFYEzrmU0rEjvP9+2FGUumKTgqr+APQKuqWKqm6O8b0bAcujHmcDPfPtcziAiEwGMoBbVfV/+d9IRC4DLgNoGqdW4KZNbdZc55wrkWXLYPFi6N8/7EhKRSzTXBwkIs8Cr6jqZhFpKyLDYnjvgnr65+/YWx5oCQwAzgaeEZFav3qR6lOqmqmqmfXq1Yvh0CW3ahVs2xaXt3bOpbNLLrEuqjnpMXwrljaFUcB7QGTy6PnEVvefDTSJetwYWFlEFZ5yAAAaRklEQVTAPm+q6i5VXQzMw5JEwl10kc155ZxzJXL55bB0KXz6adiRlIpYkkJdVR1P0C1VVXOwbqnFmQa0FJEWIlIROAuYkG+fN4CjAESkLladtCjG2EvV/PnQqlUYR3bOpbRjj4WMDJtnPw3EkhR+EZE6BFU/ItIL2Fjci4LkcRVWypgDjFfV70Xk9qjeTO8B60RkNvAx8BdVXbcP57FfNm+2RN+6daKP7JxLeTVq2GC2NGl0jqX30bXYL/xDgwbhesDQWN5cVd8B3sn33M1R9zV4/2tjDTgeFi60cQodOoQZhXMuZR19tHVR3bIFqlULO5r9UmxJQVW/AfoDfYDfA+1U9bt4B5ZIy4M+Uk2aFL2fc84VaPhwm145xRMCxLjyWjDv0feqOgsYICLpUU4KDB4Mf/2rVx855/ZR3bpQ61cdJ1NSoUlBRI4WkfkiskVEXgq6omYB/wAeT1yI8bNpEyxaBBUrwj33QPW0nObPOZcQY8bAFVeEHcV+K6qkcD82YKwO8CowBXhRVbup6muJCC6ecnLg1FNtvImPT3DO7beFC+HJJ1N+LqSikoKq6iRV3aGqbwBrVPXhRAUWb089BR9/DHfcAVWqhB2Ncy7lDRxoPVYmTgw7kv1SVO+jWiIyJOqxRD9O5dLChg02FfqAATYQ0Tnn9lvPnlC/Prz+Opx1VtjR7LOiksIn7L3CWvRjBVI2KTz4IKxbBw884MtuOudKSUaG1UmPHm110ilaBVFoUlDVixMZSCL9+COcdhp06RJ2JM65tDJ0KCxYAD/9BM2bhx3NPhFNscWnMzMzNSsra7/fR9VLCc65skNEvlbVzOL2K3MLT27YYCU7TwjOubhZuxZ27ix+vyRU5pLCgQfCAQeEHYVzLm19/jk0aABffhl2JPukxElBRDJFpFE8gom37dvtb6dO4cbhnEtj7dvDnj0waVLYkeyTfSkpDAfeEpFxpR1MvM0OFgK96aZw43DOpbFataBXL5iQf6WA1FDipKCqF6pqF+B3cYgnrmbNsr8+G6pzLq4GD4Zvv4U1a8KOpMRiSgoi0khE+ohIv8itBGs1J435860r8aGHhh2Jcy6tnXyydXF8882wIymxYtdTEJF7gDOB2eStuKZAyq0916sXjBgBFSqEHYlzLq116mTzIB17bNiRlFix4xREZB7QUVV3JCakopXWOAXnnCtLSnOcwiIgLX5bZ2fDrl1hR+GcKxM2boQXX4QlS8KOpERiSQpbgeki8qSIPBK5xTuw0paTA82a2ayozjkXd+vXwwUXwHvvhR1JicSyRvOE4JbSVq2yrsONG4cdiXOuTGje3Lqnfvtt2JGUSLFJQVWfF5GKwOHBU/NUNeUqYVatsr8HHxxuHM65MkIEMjPhq6/CjqREiq0+EpEBwAJgJPAYMF9E+sU5rlK3dq39rVcv3Dicc2VI377w3Xe29m+KiKVN4X7gWFXtr6r9gN8AD8Y3rNIXWSGvdu1w43DOlSFHHmnjFaZMCTuSmMXSplBBVedFHqjqfBFJud5IXbvarUGDsCNxzpUZ/frBypUpVW8dS1LIEpFngReDx+cCX8cvpPho08bWZK5RI+xInHNlRoUKKZUQILbqoyuA74GrgT9iI5svj2dQ8SDiCcE5F4KsLFumM0XmQSo2KajqDlV9QFWHqOppqvpgsoxuds65lPDmmzBxYthRxKTQpCAi44O/M0VkRv5b4kJ0zrkU1qWLVVN8/nnYkcSkqDaFPwZ/T0xEIM45l5YyMqB3b2vUTIHF4QstKahqMNyLtcByVV0KVAI6ASsTEJtzzqWHU06BefPyFnVJYrE0NH8KVA6W4PwQuBgYFc+gnHMurZx+OnTrBj//HHYkxYqlS6qo6lYRGQb8S1XvFZHUmszDOefCVK+e9UJKAbGUFEREemPjE94OnoslmTjnnIu2aZNVIyWxWC7u1wA3AK+r6vcicgjwcXzDcs65NHT00VC+PHz5ZdI2OMcyTuETVT1ZVe8JHi9S1avjH5pzzqWZ3/3OZk197bWwIylUUeMUHgr+/ldEJuS/xfLmIjJYROaJyEIRGVHEfqeLiIpIsUvFOedcyho2zMYtXH65zYmUhIqqPorMdXTfvryxiGRg020PArKBaSIyQVVn59uvOjaFRmpNOu6ccyVVoQK8/DJ07gzXXQdjxoQd0a8UmhRUNTLpXRawTVX3QO7FvlIM790DWKiqi4LXjQVOweZOinYHcC/w55KF7pxzKah1axgxwlZk27kTKlYMO6K9xNL76EPggKjHVYAPYnhdI2B51OPs4LlcItIFaKKqbxX1RiJymYhkiUjWmhSZVMo55wr1f/8Hb7yRdAkBYksKlVV1S+RBcP+AIvaPKKhpXXM3ipTDFuu5rrg3UtWnVDVTVTPr+dJpzrlUl5FhvY/mzEm6RudYksIvItI18kBEugHbYnhdNtAk6nFj9p4eozrQHpgkIkuAXsAEb2x2zpUZN95ojc+bN4cdSa5YksI1wCsi8pmIfAaMA66K4XXTgJYi0kJEKgJnAbm9llR1o6rWVdXmqtocmAKcrKqpMezPOef21w03wIYN1vicJGIZpzANaI0ttnMl0CaqEbqo1+VgyeM9YA4wPhj8druInLx/YTvnXBro3h2aNoUPYmmmTYxiRzSLyAHAtUAzVb1URFqKSKviGocBVPUd4J18z91cyL4DYgvZOefShAgcdRS89Rbs2QPlYqm8ia9YIvg3sBPoHTzOBu6MW0TOOVeWHHWUdU1dtizsSIDYksKhqnovsAtAVbdRcM8i55xzJXXWWbB+PTRvHnYkQGwT4u0UkSoE3UlF5FDA12h2zrnSUCmWscCJE0tJ4Rbgf0ATERmNDWb7a1yjcs65suTRR+Gcc8KOAigmKYiIAHOBIcBFwBggU1UnxT0y55wrK1asgFdegZycsCMpOimoqgJvqOo6VX1bVd9S1bUJis0558qGVq0sISxeHHYkMVUfTRGR7nGPxDnnyqpWrexvEqzKFktSOApLDD+IyAwRmSkiM+IdmHPOlRmRpDB3brhxEFvvo+PiHoVzzpVltWtDnz5QuXLYkRSeFESkMnA5cBgwE3g2mLrCOedcaZs8OewIgKKrj54HMrGEcBxwf0Iics45F5qikkJbVT1PVZ8ETgeOTFBMzjlX9rzyCjRpAuvWhRpGUUlhV+SOVxs551ycHXAAZGeH3gOpqKTQSUQ2BbfNQMfIfRHZlKgAnXOuTGjd2v6G3AOp0IZmVc1IZCDOOVemNW9uazYncUnBOedcomRkQLNmsGRJqGHEMk7BOedcIpx+OtSqFWoInhSccy5Z/P3vYUfg1UfOOZdUdu2ypTlD4knBOeeSxX/+Y11TFy0KLQRPCs45lyxq1LAptFetCi0ETwrOOZcsGja0vytXhhaCJwXnnEsWnhScc87lqlULKlUKtfrIu6Q651yyEIHrroPMzNBC8KTgnHPJ5K67Qj28Vx8551wy2b0bVq8O7fCeFJxzLplcdx0cemhoh/ek4JxzyaRpU9iyBdavD+XwnhSccy6ZHHyw/f3xx1AO70nBOeeSyUEH2d+ffgrl8J4UnHMumdSpY39DWqvZk4JzziWTJk1sCu127UI5vI9TcM65ZFK7NtxwQ2iHj2tJQUQGi8g8EVkoIiMK2H6tiMwWkRki8qGINItnPM45lxKWLg1tqou4JQURyQBGAscBbYGzRaRtvt2+BTJVtSPwKnBvvOJxzrmU0a0b3HFHKIeOZ0mhB7BQVRep6k5gLHBK9A6q+rGqbg0eTgEaxzEe55xLDQceCD//HMqh45kUGgHLox5nB88VZhjwbkEbROQyEckSkaw1a9aUYojOOZeE0jQpSAHPaYE7ipwHZAL/LGi7qj6lqpmqmlmvXr1SDNE555JQrVqwYUMoh45nUsgGmkQ9bgz8auUIETkGuAk4WVV3xDEe55xLDSGWFOLZJXUa0FJEWgArgLOAc6J3EJEuwJPAYFUNb1pA55xLJpddBmvXhnLouCUFVc0RkauA94AM4DlV/V5EbgeyVHUCVl1UDXhFRACWqerJ8YrJOedSwsCBoR06roPXVPUd4J18z90cdf+YeB7fOedS0tq1MH8+dO8OFSok9NBpMaJ5165dZGdns3379rBDSUmVK1emcePGVEjwPz7nXCFeew1+/3tYvhwaJ7anflokhezsbKpXr07z5s0JqqFcjFSVdevWkZ2dTYsWLcIOxzkH1tAM1gMpwUkhLSbE2759O3Xq1PGEsA9EhDp16ngpy7lkEkkKIfRASoukAHhC2A/+2TmXZDwpOOecy1Wrlv31pJC6MjIy6Ny5M+3bt2fo0KFs3bq1+BcVIysri6uvvrrQ7StXruT000/f7+M455JMw4Ywdiz075/wQ3tSKCVVqlRh+vTpzJo1i4oVK/LEE0/stV1V2bNnT4neMzMzk0ceeaTQ7Q0bNuTVV1/dp3idc0msShU480xo3jzhh07LpDBgwK9vjz1m27ZuLXj7qFG2fe3aX28rqSOPPJKFCxeyZMkS2rRpw5VXXknXrl1Zvnw5EydOpHfv3nTt2pWhQ4eyZcsWAKZNm0afPn3o1KkTPXr0YPPmzUyaNIkTTzwRgE8++YTOnTvTuXNnunTpwubNm1myZAnt27cHrLH94osvpkOHDnTp0oWPP/4YgFGjRjFkyBAGDx5My5Yt+etf/1ryE3LOJd5nn8F33yX8sGmZFMKUk5PDu+++S4cOHQCYN28eF1xwAd9++y1Vq1blzjvv5IMPPuCbb74hMzOTBx54gJ07d3LmmWfy8MMP89133/HBBx9QpUqVvd73vvvuY+TIkUyfPp3PPvvsV9tHjhwJwMyZMxkzZgwXXnhhbo+i6dOnM27cOGbOnMm4ceNYvnw5zrkkd/75cP/9CT9sWoxTyG/SpMK3HXBA0dvr1i16e2G2bdtG586dASspDBs2jJUrV9KsWTN69eoFwJQpU5g9ezZ9+/YFYOfOnfTu3Zt58+Zx8MEH0717dwBq1Kjxq/fv27cv1157Leeeey5Dhgyhcb6+y59//jnDhw8HoHXr1jRr1oz58+cDMHDgQGrWrAlA27ZtWbp0KU2aNME5l8Rq1YKNGxN+2LRMCmGItCnkV7Vq1dz7qsqgQYMYM2bMXvvMmDGj2G6hI0aM4IQTTuCdd96hV69efPDBB1SuXHmv9y5MpUqVcu9nZGSQk5NT7Pk450JWs6b3Pkp3vXr1YvLkySxcuBCArVu3Mn/+fFq3bs3KlSuZNm0aAJs3b/7VhfuHH36gQ4cOXH/99WRmZjJ37ty9tvfr14/Ro0cDMH/+fJYtW0arVq0ScFbOubg48MBQSgqeFBKoXr16jBo1irPPPpuOHTvSq1cv5s6dS8WKFRk3bhzDhw+nU6dODBo06FcjjB966CHat29Pp06dqFKlCscdd9xe26+88kp2795Nhw4dOPPMMxk1atReJQTnXIoJqfpIiqp2SEaZmZmalZW113Nz5syhTZs2IUWUHvwzdC7JzJoFmzZBnz6l8nYi8rWqZha3n7cpOOdcMgq6myeaVx8551wyWrwYXn4Ztm1L6GE9KTjnXDL66CM491xYsyahh/Wk4JxzySgyXmnTpoQe1pOCc84lo0hSSHAPJE8KzjmXjCLTZ3tSSE3RU2efdNJJbNiwoVTff9SoUVx11VUA3Hrrrdx3332l+v7OuSQTSQqlfC0pjieFUhI9dXbt2rVzJ6hzzrl90qwZfPUV5BuoGm/pOU6hoPmuzzgDrrzS5s4+/vhfb7/oIrutXQv5F64p4Qx5vXv3ZsaMGbmP//nPfzJ+/Hh27NjBaaedxm233QbACy+8wH333YeI0LFjR1588UX++9//cuedd7Jz507q1KnD6NGjOeigg0p0fOdcGqhcGXr0SPhh0zMphGj37t18+OGHDBs2DICJEyeyYMECpk6diqpy8skn8+mnn1KnTh3uuusuJk+eTN26dVm/fj0ARxxxBFOmTEFEeOaZZ7j33nu5P4Tpc51zSeDll6F+fTjmmIQdMj2TQghzZ0emzl6yZAndunVj0KBBgCWFiRMn0qVLFwC2bNnCggUL+O677zj99NOpW7cuALVr1wYgOzubM888k1WrVrFz505atGhR4licc2nillugW7eEJgVvUyglkTaFpUuXsnPnztw2BVXlhhtuYPr06UyfPp2FCxcybNgwVLXA6bKHDx/OVVddxcyZM3nyySd/NTGec64MOeggWL06oYf0pFDKatasySOPPMJ9993Hrl27+M1vfsNzzz2Xu+zmihUrWL16NQMHDmT8+PGsW7cOILf6aOPGjTRq1AiA559/PpyTcM4lhwYNYNWqhB4yPauPQtalSxc6derE2LFjOf/885kzZw69e/cGoFq1arz00ku0a9eOm266if79+5ORkUGXLl0YNWoUt956K0OHDqVRo0b06tWLxYsXh3w2zrnQHHwwvP8+qEIxC3GVFp862wH+GTqXlJ5+Gi67DBYsgMMO26+3inXqbK8+cs65ZHXGGbBy5X4nhJLw6iPnnEtWNWvaDWD3bsjIiPsh06akkGrVYMnEPzvnktiWLbb62kMPJeRwaZEUKleuzLp16/zitg9UlXXr1lG5cuWwQ3HOFaRaNahUCR59NCGHS4vqo8aNG5Odnc2aBC9GkS4qV65M48aNww7DOVeYU0+Fa66x7qkHHxzXQ6VFUqhQoYKP/HXOpa+OHe3vrFlxTwpxrT4SkcEiMk9EForIiAK2VxKRccH2r0SkeTzjcc65lBQpySdgIFvckoKIZAAjgeOAtsDZItI2327DgJ9V9TDgQeCeeMXjnHMpq359+P3voXnzuB8qntVHPYCFqroIQETGAqcAs6P2OQW4Nbj/KvCoiIh6i7FzzuWpWROeeCIhh4pnUmgELI96nA30LGwfVc0RkY1AHWBt9E4ichlwWfBwi4jMK0EcdfO/XxlRFs+7LJ4zlM3zLovnDPt33s1i2SmeSaGgiTrylwBi2QdVfQp4ap+CEMmKZWh3uimL510WzxnK5nmXxXOGxJx3PBuas4EmUY8bAysL20dEygM1gfVxjMk551wR4pkUpgEtRaSFiFQEzgIm5NtnAnBhcP904CNvT3DOufDErfooaCO4CngPyACeU9XvReR2IEtVJwDPAi+KyEKshHBWHELZp2qnNFAWz7ssnjOUzfMui+cMCTjvlJs62znnXPykxdxHzjnnSocnBeecc7nSJimUxSk1Yjjna0VktojMEJEPRSSmfsrJrrjzjtrvdBFREUn5rouxnLOInBF839+LyMuJjjEeYvg33lREPhaRb4N/58eHEWdpEpHnRGS1iMwqZLuIyCPBZzJDRLqWagCqmvI3rCH7B+AQoCLwHdA23z5XAk8E988CxoUddwLO+SjggOD+Fal+zrGed7BfdeBTYAqQGXbcCfiuWwLfAgcGj+uHHXeCzvsp4IrgfltgSdhxl8J59wO6ArMK2X488C42zqsX8FVpHj9dSgq5U2qo6k4gMqVGtFOA54P7rwIDRRK0EnZ8FHvOqvqxqm4NHk7Bxoqkuli+a4A7gHuB7YkMLk5iOedLgZGq+jOAqq5OcIzxEMt5K1AjuF+TX4+FSjmq+ilFj9c6BXhBzRSgloiU2tSp6ZIUCppSo1Fh+6hqDhCZUiNVxXLO0YZhvy5SXbHnLSJdgCaq+lYiA4ujWL7rw4HDRWSyiEwRkcEJiy5+YjnvW4HzRCQbeAcYnpjQQlXS//slkhbrKVCKU2qkkJjPR0TOAzKB/nGNKDGKPG8RKYfNuHtRogJKgFi+6/JYFdIArET4mYi0V9UNcY4tnmI577OBUap6v4j0xsY9tVfVPfEPLzRxvZalS0mhLE6pEcs5IyLHADcBJ6vqjgTFFk/FnXd1oD0wSUSWYHWuE1K8sTnWf99vquouVV0MzMOSRCqL5byHAeMBVPVLoDI2aVw6i+n//r5Kl6RQFqfUKPacg2qUJ7GEkA51zFDMeavqRlWtq6rNVbU51pZysqpmhRNuqYjl3/cbWMcCRKQuVp20KKFRlr5YznsZMBBARNpgSSHd1+WdAFwQ9ELqBWxU1VJbfSctqo80eabUSJgYz/mfQDXglaBNfZmqnhxa0KUgxvNOKzGe83vAsSIyG9gN/EVV14UX9f6L8byvA54WkT9hVSgXpfiPPURkDFYNWDdoK7kFqACgqk9gbSfHAwuBrcDFpXr8FP/8nHPOlaJ0qT5yzjlXCjwpOOecy+VJwTnnXC5PCs4553J5UnDOOZfLk4IrM0SkjohMD24/isiK4P6GoCtnaR9vgIiUaKoNEZlU0EA7EblIRB4tveicK5gnBVdmqOo6Ve2sqp2BJ4AHg/udgWKnRQhGwjuX1jwpOGcyROTpYC2CiSJSBXJ/uf9dRD4B/igi9UTkPyIyLbj1DfbrH1UK+VZEqgfvW01EXhWRuSIyOjIzr4gMDPabGcyfXyl/QCJysYjMD47dN0GfgyvjPCk4Z1piU0+3AzYAv43aVktV+6vq/cDDWAmje7DPM8E+fwb+EJQ8jgS2Bc93Aa7B5vo/BOgrIpWBUcCZqtoBm1ngiuhggqmQb8OSwaDg9c7FnScF58xiVZ0e3P8aaB61bVzU/WOAR0VkOjYHTY2gVDAZeEBErsaSSE6w/1RVzQ5m7ZwevG+r4Hjzg32exxZWidYTmKSqa4K1BMbhXAJ4HalzJnoG2d1AlajHv0TdLwf0VtVt7O0fIvI2NifNlGB22oLetzwFT31cEJ+DxiWclxScK5mJwFWRByLSOfh7qKrOVNV7gCygdRHvMRdoLiKHBY/PBz7Jt89XwICgx1QFYGhpnYBzRfGk4FzJXA1kBgumzwYuD56/RkRmich3WHtCoavcqep2bGbLV0RkJtbz6Yl8+6zCVhX7EvgA+Ka0T8S5gvgsqc4553J5ScE551wuTwrOOedyeVJwzjmXy5OCc865XJ4UnHPO5fKk4JxzLpcnBeecc7n+H7lBcc3mShU9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precision, recall, thresholds = precision_recall_curve(y_test_valid, probs_y[:, 1]) \n",
    "pr_auc = metrics.auc(recall, precision)\n",
    "\n",
    "plt.title(\"Precision-Recall vs Threshold Chart\")\n",
    "plt.plot(thresholds, precision[: -1], \"b--\", label=\"Precision\")\n",
    "plt.plot(thresholds, recall[: -1], \"r--\", label=\"Recall\")\n",
    "plt.ylabel(\"Precision, Recall\")\n",
    "plt.xlabel(\"Threshold\")\n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.ylim([0,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We pick thrshold of 0.3 (宁杀错，不放过）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_probs = lr.predict_proba(X_test_valid)[:, 1]\n",
    "fpr, tpr, _ = metrics.roc_curve(y_test_valid, lr_probs)\n",
    "auc = metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9485906756435661"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_probs = lr.predict_proba(X_test)[:, 1]\n",
    "fpr, tpr, _ = metrics.roc_curve(y_test, lr_probs)\n",
    "auc = metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.863611910272487"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
